In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn import metrics
from sklearn.model_selection import ParameterGrid
from sklearn.cluster import KMeans
from sklearn.preprocessing import LabelEncoder
In [2]:
def load_embeddings():
"""
Dataset load in pandas dataframe
:return: scaled & encoded data
"""
# loading dataset
dataset = pd.read_csv("lab_courses_2015.csv", sep=';')
# checking data shape
row, col = dataset.shape
print(f'There are {row} rows and {col} columns')
print(dataset.head(10))
# to work on copy of the data
dataset_scaled = dataset.copy()
# Scaling the data to keep the different attributes in same range.
dataset_scaled[dataset_scaled.columns] = StandardScaler().fit_transform(dataset_scaled)
print(dataset_scaled.describe())
return dataset_scaled
In [3]:
def pca_embeddings(df_scaled):
"""Reduce the dimensions of dataset with use of Principal Component Analysis (PCA).
:param df_scaled: scaled data
:return: pca result, pca for plotting graph
"""
pca_2 = PCA(n_components=2)
pca_2_result = pca_2.fit_transform(df_scaled)
print('Explained variation per principal component: {}'.format(pca_2.explained_variance_ratio_))
print('Cumulative variance explained by 2 principal components: {:.2%}'.format(
np.sum(pca_2.explained_variance_ratio_)))
# Results from pca.components_
dataset_pca = pd.DataFrame(abs(pca_2.components_), columns=df_scaled.columns, index=['PC_1', 'PC_2'])
print('\n\n', dataset_pca)
print("\n*************** Most important features *************************")
print('As per PC 1:\n', (dataset_pca[dataset_pca > 0.3].iloc[0]).dropna())
print('\n\nAs per PC 2:\n', (dataset_pca[dataset_pca > 0.3].iloc[1]).dropna())
print("\n******************************************************************")
return pca_2_result, pca_2
In [4]:
def visualizing_results(pca_result, label, centroids_pca):
""" Visualizing the clusters
:param pca_result: PCA applied data
:param label: K Means labels
:param centroids_pca: PCA format K Means centroids
"""
# ------------------ Using Matplotlib for plotting-----------------------
x = pca_result[:, 0]
y = pca_result[:, 1]
plt.figure(figsize=(14,7), dpi=1500)
plt.scatter(x, y, c=label, alpha=0.6, s=200, edgecolors="black") # plot different colors per cluster
plt.title('Student clusters')
plt.xlabel('First Principal Component')
plt.ylabel('Second Principal Component')
plt.scatter(centroids_pca[:, 0], centroids_pca[:, 1], marker='X', s=200, linewidths=1.5,
color='red', edgecolors="black", lw=1.5)
plt.show()
plt.savefig('clusters.png')
In [5]:
def main():
print("1. Loading dataset\n")
data_scaled = load_embeddings()
print("\n\n2. Reducing via PCA\n")
pca_result, pca_2 = pca_embeddings(data_scaled)
# fitting KMeans
kmeans = KMeans(n_clusters=4)
kmeans.fit(data_scaled)
centroids = kmeans.cluster_centers_
centroids_pca = pca_2.transform(centroids)
print("\n\n4. Visualizing the data")
visualizing_results(pca_result, kmeans.labels_, centroids_pca)
if __name__ == "__main__":
main()
